{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Installation\\anaconda\\install\\lib\\site-packages\\dask\\config.py:168: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  data = yaml.load(f.read()) or {}\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "import catboost as cb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(57081, 11)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"flights.csv\")\n",
    "data = data.sample(frac=0.01, random_state=10)\n",
    "\n",
    "data = data[[\"MONTH\",\"DAY\",\"DAY_OF_WEEK\",\"AIRLINE\",\"FLIGHT_NUMBER\",\"DESTINATION_AIRPORT\",\n",
    "                 \"ORIGIN_AIRPORT\",\"AIR_TIME\", \"DEPARTURE_TIME\",\"DISTANCE\",\"ARRIVAL_DELAY\"]]\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MONTH</th>\n",
       "      <th>DAY</th>\n",
       "      <th>DAY_OF_WEEK</th>\n",
       "      <th>AIRLINE</th>\n",
       "      <th>FLIGHT_NUMBER</th>\n",
       "      <th>DESTINATION_AIRPORT</th>\n",
       "      <th>ORIGIN_AIRPORT</th>\n",
       "      <th>AIR_TIME</th>\n",
       "      <th>DEPARTURE_TIME</th>\n",
       "      <th>DISTANCE</th>\n",
       "      <th>ARRIVAL_DELAY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>102</td>\n",
       "      <td>516</td>\n",
       "      <td>413</td>\n",
       "      <td>102.0</td>\n",
       "      <td>713.0</td>\n",
       "      <td>634</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>152</td>\n",
       "      <td>547</td>\n",
       "      <td>490</td>\n",
       "      <td>134.0</td>\n",
       "      <td>111.0</td>\n",
       "      <td>1028</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>1184</td>\n",
       "      <td>399</td>\n",
       "      <td>539</td>\n",
       "      <td>111.0</td>\n",
       "      <td>1734.0</td>\n",
       "      <td>931</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>170</td>\n",
       "      <td>568</td>\n",
       "      <td>414</td>\n",
       "      <td>173.0</td>\n",
       "      <td>1807.0</td>\n",
       "      <td>1436</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "      <td>4151</td>\n",
       "      <td>570</td>\n",
       "      <td>349</td>\n",
       "      <td>63.0</td>\n",
       "      <td>2151.0</td>\n",
       "      <td>481</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   MONTH  DAY  DAY_OF_WEEK  AIRLINE  FLIGHT_NUMBER  DESTINATION_AIRPORT  \\\n",
       "0      1   28            3       14            102                  516   \n",
       "1      8   11            2        3            152                  547   \n",
       "2      2    4            3        4           1184                  399   \n",
       "3      3   27            5       14            170                  568   \n",
       "4      8    1            6       14           4151                  570   \n",
       "\n",
       "   ORIGIN_AIRPORT  AIR_TIME  DEPARTURE_TIME  DISTANCE  ARRIVAL_DELAY  \n",
       "0             413     102.0           713.0       634              0  \n",
       "1             490     134.0           111.0      1028              1  \n",
       "2             539     111.0          1734.0       931              0  \n",
       "3             414     173.0          1807.0      1436              0  \n",
       "4             349      63.0          2151.0       481              1  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.reset_index(drop=True)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(39956, 10) (39956,) (17125, 10) (17125,)\n"
     ]
    }
   ],
   "source": [
    "data[\"ARRIVAL_DELAY\"] = (data[\"ARRIVAL_DELAY\"]>10)*1\n",
    "\n",
    "cols = [\"AIRLINE\",\"FLIGHT_NUMBER\",\"DESTINATION_AIRPORT\",\"ORIGIN_AIRPORT\"]\n",
    "for item in cols:\n",
    "    data[item] = data[item].astype(\"category\").cat.codes +1\n",
    "    \n",
    "X_train, X_test, y_train, y_test = train_test_split(data.drop([\"ARRIVAL_DELAY\"], axis=1), data[\"ARRIVAL_DELAY\"],\n",
    "                                                random_state=10, test_size=0.3)\n",
    "print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    31248\n",
       "1     8708\n",
       "Name: ARRIVAL_DELAY, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    13458\n",
       "1     3667\n",
       "Name: ARRIVAL_DELAY, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[23:25:38] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "training spend 20.950793504714966 seconds\n",
      "testing spend 0.23138022422790527 seconds\n",
      "0.7516480044157828\n",
      "0.6845368959487046\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score\n",
    "\n",
    "# 设置模型参数\n",
    "params = {\n",
    "    'booster': 'gbtree',\n",
    "    'objective': 'binary:logistic',   \n",
    "    'gamma': 0.1,\n",
    "    'max_depth': 8,\n",
    "    'lambda': 2,\n",
    "    'subsample': 0.7,\n",
    "    'colsample_bytree': 0.7,\n",
    "    'min_child_weight': 3,\n",
    "    'eta': 0.001,\n",
    "    'seed': 1000,\n",
    "    'nthread': 4,\n",
    "}\n",
    "t0 = time.time()\n",
    "# plst = params.items()\n",
    "num_rounds = 500\n",
    "dtrain = xgb.DMatrix(X_train, y_train)\n",
    "model_xgb = xgb.train(params, dtrain, num_rounds)\n",
    "print('training spend {} seconds'.format(time.time()-t0))\n",
    "# 对测试集进行预测\n",
    "t1 = time.time()\n",
    "dtest = xgb.DMatrix(X_test)\n",
    "y_pred = model_xgb.predict(dtest)\n",
    "print('testing spend {} seconds'.format(time.time()-t1))\n",
    "y_pred_train = model_xgb.predict(dtrain)\n",
    "print(roc_auc_score(y_train, y_pred_train))\n",
    "print(roc_auc_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Installation\\anaconda\\install\\lib\\site-packages\\lightgbm\\engine.py:151: UserWarning: Found `n_estimators` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n",
      "D:\\Installation\\anaconda\\install\\lib\\site-packages\\lightgbm\\basic.py:1555: UserWarning: categorical_feature in Dataset is overridden.\n",
      "New categorical_feature is ['AIRLINE', 'DAY', 'DAY_OF_WEEK', 'DESTINATION_AIRPORT', 'MONTH', 'ORIGIN_AIRPORT']\n",
      "  'New categorical_feature is {}'.format(sorted(list(categorical_feature))))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.007086 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 1868\n",
      "[LightGBM] [Info] Number of data points in the train set: 39956, number of used features: 10\n",
      "[LightGBM] [Info] Start training from score 0.217940\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "[LightGBM] [Warning] No further splits with positive gain, best gain: -inf\n",
      "training spend 2.1756861209869385 seconds\n",
      "testing spend 0.5056381225585938 seconds\n",
      "0.8812196631020766\n",
      "0.6873707383550387\n"
     ]
    }
   ],
   "source": [
    "d_train = lgb.Dataset(X_train, label=y_train)\n",
    "params = {\"max_depth\": 5, \"learning_rate\" : 0.05, \"num_leaves\": 500,  \"n_estimators\": 300}\n",
    "\n",
    "#With Catgeorical Features\n",
    "cate_features_name = [\"MONTH\",\"DAY\",\"DAY_OF_WEEK\",\"AIRLINE\",\"DESTINATION_AIRPORT\",\n",
    "                 \"ORIGIN_AIRPORT\"]\n",
    "t0 = time.time()\n",
    "model_lgb = lgb.train(params, d_train, categorical_feature = cate_features_name)\n",
    "print('training spend {} seconds'.format(time.time()-t0))\n",
    "t1 = time.time()\n",
    "y_pred = model_lgb.predict(X_test)\n",
    "print('testing spend {} seconds'.format(time.time()-t1))\n",
    "y_pred_train = model_lgb.predict(X_train)\n",
    "print(roc_auc_score(y_train, y_pred_train))\n",
    "print(roc_auc_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\ttotal: 98.3ms\tremaining: 29.4s\n",
      "1:\ttotal: 199ms\tremaining: 29.7s\n",
      "2:\ttotal: 298ms\tremaining: 29.5s\n",
      "3:\ttotal: 428ms\tremaining: 31.6s\n",
      "4:\ttotal: 544ms\tremaining: 32.1s\n",
      "5:\ttotal: 657ms\tremaining: 32.2s\n",
      "6:\ttotal: 756ms\tremaining: 31.7s\n",
      "7:\ttotal: 838ms\tremaining: 30.6s\n",
      "8:\ttotal: 945ms\tremaining: 30.6s\n",
      "9:\ttotal: 1.08s\tremaining: 31.3s\n",
      "10:\ttotal: 1.2s\tremaining: 31.6s\n",
      "11:\ttotal: 1.34s\tremaining: 32.1s\n",
      "12:\ttotal: 1.46s\tremaining: 32.2s\n",
      "13:\ttotal: 1.58s\tremaining: 32.2s\n",
      "14:\ttotal: 1.62s\tremaining: 30.8s\n",
      "15:\ttotal: 1.77s\tremaining: 31.4s\n",
      "16:\ttotal: 1.87s\tremaining: 31.1s\n",
      "17:\ttotal: 2s\tremaining: 31.4s\n",
      "18:\ttotal: 2.21s\tremaining: 32.6s\n",
      "19:\ttotal: 2.35s\tremaining: 32.9s\n",
      "20:\ttotal: 2.51s\tremaining: 33.3s\n",
      "21:\ttotal: 2.63s\tremaining: 33.2s\n",
      "22:\ttotal: 2.76s\tremaining: 33.3s\n",
      "23:\ttotal: 2.88s\tremaining: 33.1s\n",
      "24:\ttotal: 2.99s\tremaining: 32.9s\n",
      "25:\ttotal: 3.14s\tremaining: 33.1s\n",
      "26:\ttotal: 3.31s\tremaining: 33.5s\n",
      "27:\ttotal: 3.46s\tremaining: 33.6s\n",
      "28:\ttotal: 3.61s\tremaining: 33.8s\n",
      "29:\ttotal: 3.77s\tremaining: 33.9s\n",
      "30:\ttotal: 3.93s\tremaining: 34.1s\n",
      "31:\ttotal: 4.06s\tremaining: 34s\n",
      "32:\ttotal: 4.2s\tremaining: 34s\n",
      "33:\ttotal: 4.34s\tremaining: 34s\n",
      "34:\ttotal: 4.47s\tremaining: 33.9s\n",
      "35:\ttotal: 4.58s\tremaining: 33.6s\n",
      "36:\ttotal: 4.69s\tremaining: 33.3s\n",
      "37:\ttotal: 4.88s\tremaining: 33.6s\n",
      "38:\ttotal: 5.13s\tremaining: 34.3s\n",
      "39:\ttotal: 5.26s\tremaining: 34.2s\n",
      "40:\ttotal: 5.52s\tremaining: 34.8s\n",
      "41:\ttotal: 5.68s\tremaining: 34.9s\n",
      "42:\ttotal: 5.94s\tremaining: 35.5s\n",
      "43:\ttotal: 6.21s\tremaining: 36.1s\n",
      "44:\ttotal: 6.34s\tremaining: 36s\n",
      "45:\ttotal: 6.46s\tremaining: 35.7s\n",
      "46:\ttotal: 6.59s\tremaining: 35.5s\n",
      "47:\ttotal: 6.76s\tremaining: 35.5s\n",
      "48:\ttotal: 6.94s\tremaining: 35.6s\n",
      "49:\ttotal: 7.11s\tremaining: 35.5s\n",
      "50:\ttotal: 7.22s\tremaining: 35.3s\n",
      "51:\ttotal: 7.32s\tremaining: 34.9s\n",
      "52:\ttotal: 7.44s\tremaining: 34.7s\n",
      "53:\ttotal: 7.56s\tremaining: 34.4s\n",
      "54:\ttotal: 7.69s\tremaining: 34.2s\n",
      "55:\ttotal: 7.79s\tremaining: 34s\n",
      "56:\ttotal: 7.92s\tremaining: 33.8s\n",
      "57:\ttotal: 8.02s\tremaining: 33.5s\n",
      "58:\ttotal: 8.14s\tremaining: 33.3s\n",
      "59:\ttotal: 8.27s\tremaining: 33.1s\n",
      "60:\ttotal: 8.39s\tremaining: 32.9s\n",
      "61:\ttotal: 8.5s\tremaining: 32.6s\n",
      "62:\ttotal: 8.62s\tremaining: 32.4s\n",
      "63:\ttotal: 8.72s\tremaining: 32.2s\n",
      "64:\ttotal: 8.86s\tremaining: 32s\n",
      "65:\ttotal: 8.98s\tremaining: 31.8s\n",
      "66:\ttotal: 9.09s\tremaining: 31.6s\n",
      "67:\ttotal: 9.21s\tremaining: 31.4s\n",
      "68:\ttotal: 9.32s\tremaining: 31.2s\n",
      "69:\ttotal: 9.42s\tremaining: 31s\n",
      "70:\ttotal: 9.58s\tremaining: 30.9s\n",
      "71:\ttotal: 9.87s\tremaining: 31.2s\n",
      "72:\ttotal: 9.98s\tremaining: 31s\n",
      "73:\ttotal: 10.3s\tremaining: 31.4s\n",
      "74:\ttotal: 10.4s\tremaining: 31.1s\n",
      "75:\ttotal: 10.5s\tremaining: 30.8s\n",
      "76:\ttotal: 10.6s\tremaining: 30.7s\n",
      "77:\ttotal: 10.7s\tremaining: 30.5s\n",
      "78:\ttotal: 10.8s\tremaining: 30.3s\n",
      "79:\ttotal: 11s\tremaining: 30.1s\n",
      "80:\ttotal: 11.1s\tremaining: 30s\n",
      "81:\ttotal: 11.2s\tremaining: 29.8s\n",
      "82:\ttotal: 11.3s\tremaining: 29.6s\n",
      "83:\ttotal: 11.4s\tremaining: 29.4s\n",
      "84:\ttotal: 11.5s\tremaining: 29.2s\n",
      "85:\ttotal: 11.6s\tremaining: 29s\n",
      "86:\ttotal: 11.8s\tremaining: 28.8s\n",
      "87:\ttotal: 11.9s\tremaining: 28.6s\n",
      "88:\ttotal: 12.1s\tremaining: 28.6s\n",
      "89:\ttotal: 12.3s\tremaining: 28.6s\n",
      "90:\ttotal: 12.4s\tremaining: 28.5s\n",
      "91:\ttotal: 12.6s\tremaining: 28.4s\n",
      "92:\ttotal: 12.7s\tremaining: 28.2s\n",
      "93:\ttotal: 12.8s\tremaining: 27.9s\n",
      "94:\ttotal: 12.9s\tremaining: 27.8s\n",
      "95:\ttotal: 13s\tremaining: 27.6s\n",
      "96:\ttotal: 13.1s\tremaining: 27.5s\n",
      "97:\ttotal: 13.2s\tremaining: 27.3s\n",
      "98:\ttotal: 13.4s\tremaining: 27.1s\n",
      "99:\ttotal: 13.5s\tremaining: 26.9s\n",
      "100:\ttotal: 13.6s\tremaining: 26.8s\n",
      "101:\ttotal: 13.7s\tremaining: 26.6s\n",
      "102:\ttotal: 13.8s\tremaining: 26.4s\n",
      "103:\ttotal: 13.9s\tremaining: 26.3s\n",
      "104:\ttotal: 14.1s\tremaining: 26.1s\n",
      "105:\ttotal: 14.1s\tremaining: 25.9s\n",
      "106:\ttotal: 14.3s\tremaining: 25.8s\n",
      "107:\ttotal: 14.4s\tremaining: 25.7s\n",
      "108:\ttotal: 14.6s\tremaining: 25.6s\n",
      "109:\ttotal: 14.8s\tremaining: 25.6s\n",
      "110:\ttotal: 15s\tremaining: 25.5s\n",
      "111:\ttotal: 15.1s\tremaining: 25.4s\n",
      "112:\ttotal: 15.2s\tremaining: 25.2s\n",
      "113:\ttotal: 15.3s\tremaining: 25s\n",
      "114:\ttotal: 15.5s\tremaining: 24.9s\n",
      "115:\ttotal: 15.6s\tremaining: 24.7s\n",
      "116:\ttotal: 15.7s\tremaining: 24.5s\n",
      "117:\ttotal: 15.8s\tremaining: 24.4s\n",
      "118:\ttotal: 15.9s\tremaining: 24.2s\n",
      "119:\ttotal: 16s\tremaining: 24.1s\n",
      "120:\ttotal: 16.2s\tremaining: 23.9s\n",
      "121:\ttotal: 16.3s\tremaining: 23.7s\n",
      "122:\ttotal: 16.4s\tremaining: 23.6s\n",
      "123:\ttotal: 16.5s\tremaining: 23.4s\n",
      "124:\ttotal: 16.6s\tremaining: 23.2s\n",
      "125:\ttotal: 16.7s\tremaining: 23.1s\n",
      "126:\ttotal: 16.8s\tremaining: 22.9s\n",
      "127:\ttotal: 16.9s\tremaining: 22.8s\n",
      "128:\ttotal: 17.1s\tremaining: 22.6s\n",
      "129:\ttotal: 17.2s\tremaining: 22.5s\n",
      "130:\ttotal: 17.3s\tremaining: 22.3s\n",
      "131:\ttotal: 17.4s\tremaining: 22.2s\n",
      "132:\ttotal: 17.5s\tremaining: 22s\n",
      "133:\ttotal: 17.7s\tremaining: 21.9s\n",
      "134:\ttotal: 17.8s\tremaining: 21.7s\n",
      "135:\ttotal: 17.9s\tremaining: 21.6s\n",
      "136:\ttotal: 18s\tremaining: 21.4s\n",
      "137:\ttotal: 18.1s\tremaining: 21.3s\n",
      "138:\ttotal: 18.3s\tremaining: 21.1s\n",
      "139:\ttotal: 18.4s\tremaining: 21s\n",
      "140:\ttotal: 18.5s\tremaining: 20.8s\n",
      "141:\ttotal: 18.6s\tremaining: 20.7s\n",
      "142:\ttotal: 18.7s\tremaining: 20.6s\n",
      "143:\ttotal: 18.9s\tremaining: 20.4s\n",
      "144:\ttotal: 19s\tremaining: 20.3s\n",
      "145:\ttotal: 19.1s\tremaining: 20.1s\n",
      "146:\ttotal: 19.2s\tremaining: 20s\n",
      "147:\ttotal: 19.3s\tremaining: 19.8s\n",
      "148:\ttotal: 19.4s\tremaining: 19.7s\n",
      "149:\ttotal: 19.5s\tremaining: 19.5s\n",
      "150:\ttotal: 19.6s\tremaining: 19.4s\n",
      "151:\ttotal: 19.8s\tremaining: 19.2s\n",
      "152:\ttotal: 19.9s\tremaining: 19.1s\n",
      "153:\ttotal: 20s\tremaining: 19s\n",
      "154:\ttotal: 20.1s\tremaining: 18.8s\n",
      "155:\ttotal: 20.2s\tremaining: 18.7s\n",
      "156:\ttotal: 20.4s\tremaining: 18.6s\n",
      "157:\ttotal: 20.5s\tremaining: 18.4s\n",
      "158:\ttotal: 20.6s\tremaining: 18.3s\n",
      "159:\ttotal: 20.7s\tremaining: 18.1s\n",
      "160:\ttotal: 20.9s\tremaining: 18s\n",
      "161:\ttotal: 21s\tremaining: 17.9s\n",
      "162:\ttotal: 21.1s\tremaining: 17.7s\n",
      "163:\ttotal: 21.2s\tremaining: 17.6s\n",
      "164:\ttotal: 21.3s\tremaining: 17.5s\n",
      "165:\ttotal: 21.5s\tremaining: 17.3s\n",
      "166:\ttotal: 21.6s\tremaining: 17.2s\n",
      "167:\ttotal: 21.7s\tremaining: 17s\n",
      "168:\ttotal: 21.8s\tremaining: 16.9s\n",
      "169:\ttotal: 21.9s\tremaining: 16.8s\n",
      "170:\ttotal: 22.1s\tremaining: 16.6s\n",
      "171:\ttotal: 22.2s\tremaining: 16.5s\n",
      "172:\ttotal: 22.3s\tremaining: 16.4s\n",
      "173:\ttotal: 22.4s\tremaining: 16.2s\n",
      "174:\ttotal: 22.5s\tremaining: 16.1s\n",
      "175:\ttotal: 22.6s\tremaining: 16s\n",
      "176:\ttotal: 22.8s\tremaining: 15.8s\n",
      "177:\ttotal: 22.9s\tremaining: 15.7s\n",
      "178:\ttotal: 23s\tremaining: 15.6s\n",
      "179:\ttotal: 23.2s\tremaining: 15.5s\n",
      "180:\ttotal: 23.3s\tremaining: 15.3s\n",
      "181:\ttotal: 23.5s\tremaining: 15.2s\n",
      "182:\ttotal: 23.6s\tremaining: 15.1s\n",
      "183:\ttotal: 23.7s\tremaining: 14.9s\n",
      "184:\ttotal: 23.8s\tremaining: 14.8s\n",
      "185:\ttotal: 23.9s\tremaining: 14.7s\n",
      "186:\ttotal: 24s\tremaining: 14.5s\n",
      "187:\ttotal: 24.1s\tremaining: 14.4s\n",
      "188:\ttotal: 24.3s\tremaining: 14.3s\n",
      "189:\ttotal: 24.4s\tremaining: 14.2s\n",
      "190:\ttotal: 24.6s\tremaining: 14s\n",
      "191:\ttotal: 24.7s\tremaining: 13.9s\n",
      "192:\ttotal: 24.8s\tremaining: 13.8s\n",
      "193:\ttotal: 24.9s\tremaining: 13.6s\n",
      "194:\ttotal: 25s\tremaining: 13.5s\n",
      "195:\ttotal: 25.2s\tremaining: 13.3s\n",
      "196:\ttotal: 25.3s\tremaining: 13.2s\n",
      "197:\ttotal: 25.4s\tremaining: 13.1s\n",
      "198:\ttotal: 25.5s\tremaining: 12.9s\n",
      "199:\ttotal: 25.6s\tremaining: 12.8s\n",
      "200:\ttotal: 25.7s\tremaining: 12.7s\n",
      "201:\ttotal: 25.8s\tremaining: 12.5s\n",
      "202:\ttotal: 25.9s\tremaining: 12.4s\n",
      "203:\ttotal: 26s\tremaining: 12.3s\n",
      "204:\ttotal: 26.2s\tremaining: 12.1s\n",
      "205:\ttotal: 26.3s\tremaining: 12s\n",
      "206:\ttotal: 26.4s\tremaining: 11.8s\n",
      "207:\ttotal: 26.5s\tremaining: 11.7s\n",
      "208:\ttotal: 26.6s\tremaining: 11.6s\n",
      "209:\ttotal: 26.7s\tremaining: 11.4s\n",
      "210:\ttotal: 26.8s\tremaining: 11.3s\n",
      "211:\ttotal: 26.9s\tremaining: 11.1s\n",
      "212:\ttotal: 27s\tremaining: 11s\n",
      "213:\ttotal: 27.1s\tremaining: 10.9s\n",
      "214:\ttotal: 27.2s\tremaining: 10.8s\n",
      "215:\ttotal: 27.3s\tremaining: 10.6s\n",
      "216:\ttotal: 27.4s\tremaining: 10.5s\n",
      "217:\ttotal: 27.5s\tremaining: 10.4s\n",
      "218:\ttotal: 27.6s\tremaining: 10.2s\n",
      "219:\ttotal: 27.7s\tremaining: 10.1s\n",
      "220:\ttotal: 27.8s\tremaining: 9.95s\n",
      "221:\ttotal: 27.9s\tremaining: 9.81s\n",
      "222:\ttotal: 28s\tremaining: 9.68s\n",
      "223:\ttotal: 28.1s\tremaining: 9.55s\n",
      "224:\ttotal: 28.3s\tremaining: 9.42s\n",
      "225:\ttotal: 28.4s\tremaining: 9.29s\n",
      "226:\ttotal: 28.5s\tremaining: 9.15s\n",
      "227:\ttotal: 28.6s\tremaining: 9.02s\n",
      "228:\ttotal: 28.7s\tremaining: 8.89s\n",
      "229:\ttotal: 28.8s\tremaining: 8.76s\n",
      "230:\ttotal: 28.9s\tremaining: 8.63s\n",
      "231:\ttotal: 29s\tremaining: 8.5s\n",
      "232:\ttotal: 29.1s\tremaining: 8.37s\n",
      "233:\ttotal: 29.2s\tremaining: 8.25s\n",
      "234:\ttotal: 29.3s\tremaining: 8.12s\n",
      "235:\ttotal: 29.4s\tremaining: 7.99s\n",
      "236:\ttotal: 29.5s\tremaining: 7.85s\n",
      "237:\ttotal: 29.7s\tremaining: 7.73s\n",
      "238:\ttotal: 29.8s\tremaining: 7.6s\n",
      "239:\ttotal: 29.9s\tremaining: 7.47s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "240:\ttotal: 30s\tremaining: 7.34s\n",
      "241:\ttotal: 30.1s\tremaining: 7.22s\n",
      "242:\ttotal: 30.2s\tremaining: 7.09s\n",
      "243:\ttotal: 30.3s\tremaining: 6.96s\n",
      "244:\ttotal: 30.4s\tremaining: 6.83s\n",
      "245:\ttotal: 30.5s\tremaining: 6.7s\n",
      "246:\ttotal: 30.6s\tremaining: 6.57s\n",
      "247:\ttotal: 30.7s\tremaining: 6.44s\n",
      "248:\ttotal: 30.8s\tremaining: 6.31s\n",
      "249:\ttotal: 30.9s\tremaining: 6.18s\n",
      "250:\ttotal: 31s\tremaining: 6.05s\n",
      "251:\ttotal: 31.1s\tremaining: 5.93s\n",
      "252:\ttotal: 31.2s\tremaining: 5.8s\n",
      "253:\ttotal: 31.3s\tremaining: 5.67s\n",
      "254:\ttotal: 31.4s\tremaining: 5.55s\n",
      "255:\ttotal: 31.5s\tremaining: 5.42s\n",
      "256:\ttotal: 31.6s\tremaining: 5.29s\n",
      "257:\ttotal: 31.7s\tremaining: 5.17s\n",
      "258:\ttotal: 31.9s\tremaining: 5.04s\n",
      "259:\ttotal: 32s\tremaining: 4.92s\n",
      "260:\ttotal: 32.1s\tremaining: 4.79s\n",
      "261:\ttotal: 32.2s\tremaining: 4.67s\n",
      "262:\ttotal: 32.3s\tremaining: 4.54s\n",
      "263:\ttotal: 32.4s\tremaining: 4.42s\n",
      "264:\ttotal: 32.5s\tremaining: 4.29s\n",
      "265:\ttotal: 32.6s\tremaining: 4.17s\n",
      "266:\ttotal: 32.7s\tremaining: 4.04s\n",
      "267:\ttotal: 32.8s\tremaining: 3.92s\n",
      "268:\ttotal: 33s\tremaining: 3.8s\n",
      "269:\ttotal: 33.1s\tremaining: 3.67s\n",
      "270:\ttotal: 33.2s\tremaining: 3.55s\n",
      "271:\ttotal: 33.3s\tremaining: 3.43s\n",
      "272:\ttotal: 33.4s\tremaining: 3.3s\n",
      "273:\ttotal: 33.5s\tremaining: 3.18s\n",
      "274:\ttotal: 33.6s\tremaining: 3.06s\n",
      "275:\ttotal: 33.7s\tremaining: 2.93s\n",
      "276:\ttotal: 33.8s\tremaining: 2.81s\n",
      "277:\ttotal: 33.9s\tremaining: 2.68s\n",
      "278:\ttotal: 34s\tremaining: 2.56s\n",
      "279:\ttotal: 34.1s\tremaining: 2.44s\n",
      "280:\ttotal: 34.2s\tremaining: 2.31s\n",
      "281:\ttotal: 34.3s\tremaining: 2.19s\n",
      "282:\ttotal: 34.4s\tremaining: 2.07s\n",
      "283:\ttotal: 34.5s\tremaining: 1.95s\n",
      "284:\ttotal: 34.6s\tremaining: 1.82s\n",
      "285:\ttotal: 34.7s\tremaining: 1.7s\n",
      "286:\ttotal: 34.9s\tremaining: 1.58s\n",
      "287:\ttotal: 35s\tremaining: 1.46s\n",
      "288:\ttotal: 35.1s\tremaining: 1.33s\n",
      "289:\ttotal: 35.2s\tremaining: 1.21s\n",
      "290:\ttotal: 35.3s\tremaining: 1.09s\n",
      "291:\ttotal: 35.4s\tremaining: 970ms\n",
      "292:\ttotal: 35.5s\tremaining: 848ms\n",
      "293:\ttotal: 35.6s\tremaining: 727ms\n",
      "294:\ttotal: 35.7s\tremaining: 605ms\n",
      "295:\ttotal: 35.8s\tremaining: 484ms\n",
      "296:\ttotal: 35.9s\tremaining: 363ms\n",
      "297:\ttotal: 36s\tremaining: 242ms\n",
      "298:\ttotal: 36.1s\tremaining: 121ms\n",
      "299:\ttotal: 36.3s\tremaining: 0us\n",
      "training spend 37.27786946296692 seconds\n",
      "testing spend 0.2802553176879883 seconds\n",
      "0.5734797895232498\n",
      "0.5463773041667715\n"
     ]
    }
   ],
   "source": [
    "cat_features_index = [0,1,2,3,4,5,6]\n",
    "t0 = time.time()\n",
    "model_cb = cb.CatBoostClassifier(eval_metric=\"AUC\", one_hot_max_size=50, \n",
    "                            depth=6, iterations=300, l2_leaf_reg=1, learning_rate=0.1)\n",
    "model_cb.fit(X_train,y_train, cat_features= cat_features_index)\n",
    "print('training spend {} seconds'.format(time.time()-t0))\n",
    "t1 = time.time()\n",
    "y_pred = model_cb.predict(X_test)\n",
    "print('testing spend {} seconds'.format(time.time()-t1))\n",
    "y_pred_train = model_cb.predict(X_train)\n",
    "print(roc_auc_score(y_train, y_pred_train))\n",
    "print(roc_auc_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_estimators': 300, 'min_child_weight': 6, 'max_depth': 5, 'learning_rate': 0.1}\n",
      "randomsearch for xgb spend 341.83500123023987 seconds.\n"
     ]
    }
   ],
   "source": [
    "### RandomSearch\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "# from scipy.stats import uniform\n",
    "model = xgb.XGBClassifier()\n",
    "param_lst = {'max_depth': [3,5,7], \n",
    "                 'min_child_weight': [1,3,6], \n",
    "                 'n_estimators': [100,200,300],\n",
    "                 'learning_rate': [0.01, 0.05, 0.1]\n",
    "                }\n",
    "t0 = time.time()\n",
    "random_search = RandomizedSearchCV(model, param_lst, random_state=0)\n",
    "random_search.fit(X_train, y_train)\n",
    "print(random_search.best_params_)\n",
    "print('randomsearch for xgb spend', time.time()-t0, 'seconds.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 81 candidates, totalling 243 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   5 tasks      | elapsed:   29.2s\n",
      "[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:   35.0s\n",
      "[Parallel(n_jobs=-1)]: Done  17 tasks      | elapsed:   46.1s\n",
      "[Parallel(n_jobs=-1)]: Done  24 tasks      | elapsed:   54.7s\n",
      "[Parallel(n_jobs=-1)]: Done  33 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done  53 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done  64 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done  77 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=-1)]: Done  90 tasks      | elapsed:  4.6min\n",
      "[Parallel(n_jobs=-1)]: Done 105 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=-1)]: Done 120 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done 137 tasks      | elapsed:  6.4min\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed:  7.5min\n",
      "[Parallel(n_jobs=-1)]: Done 173 tasks      | elapsed:  8.2min\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  8.7min\n",
      "[Parallel(n_jobs=-1)]: Done 213 tasks      | elapsed:  9.7min\n",
      "[Parallel(n_jobs=-1)]: Done 234 tasks      | elapsed: 11.0min\n",
      "[Parallel(n_jobs=-1)]: Done 243 out of 243 | elapsed: 11.5min finished\n",
      "D:\\Installation\\anaconda\\install\\lib\\site-packages\\xgboost\\sklearn.py:888: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[22:46:09] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.3.0/src/learner.cc:1061: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "gridsearch for xgb spend 696.4174864292145 seconds.\n"
     ]
    }
   ],
   "source": [
    "### GridSearch\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "model = xgb.XGBClassifier()\n",
    "param_lst = {\"max_depth\": [3,5,7],\n",
    "              \"min_child_weight\" : [1,3,6],\n",
    "              \"n_estimators\": [100,200,300],\n",
    "              \"learning_rate\": [0.01,0.05,0.1]\n",
    "             }\n",
    "t0 = time.time()\n",
    "grid_search = GridSearchCV(model, param_grid=param_lst, cv=3, \n",
    "                                   verbose=10, n_jobs=-1)\n",
    "grid_search.fit(X_train, y_train)\n",
    "print('gridsearch for xgb spend', time.time()-t0, 'seconds.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBClassifier(max_depth=5, min_child_weight=6, n_estimators=300)\n"
     ]
    }
   ],
   "source": [
    "print(grid_search.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Bayesian Opt\n",
    "from bayes_opt import BayesianOptimization\n",
    "from tqdm import tqdm\n",
    "\n",
    "def xgb_evaluate(min_child_weight,\n",
    "                 colsample_bytree,\n",
    "                 max_depth,\n",
    "                 subsample,\n",
    "                 gamma,\n",
    "                 alpha):\n",
    "\n",
    "    params['min_child_weight'] = int(min_child_weight)\n",
    "    params['cosample_bytree'] = max(min(colsample_bytree, 1), 0)\n",
    "    params['max_depth'] = int(max_depth)\n",
    "    params['subsample'] = max(min(subsample, 1), 0)\n",
    "    params['gamma'] = max(gamma, 0)\n",
    "    params['alpha'] = max(alpha, 0)\n",
    "\n",
    "\n",
    "    cv_result = xgb.cv(params, dtrain, num_boost_round=num_rounds, nfold=5,\n",
    "             seed=random_state,\n",
    "             callbacks=[xgb.callback.early_stop(50)])\n",
    "\n",
    "    return cv_result['test-auc-mean'].values[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[532]\ttrain-auc:0.79538+0.00130824\ttest-auc:0.715142+0.00480519\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-0.7151423999999998"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = xgb_evaluate(3, 0.5, 4, 0.6, 1, 1)\n",
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   |   alpha   | colsam... |   gamma   | max_depth | min_ch... | subsample |\n",
      "-------------------------------------------------------------------------------------------------\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[469]\ttrain-auc:0.811907+0.000713835\ttest-auc:0.7164+0.00529078\n",
      "\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m 0.7164  \u001b[0m | \u001b[0m 2.467   \u001b[0m | \u001b[0m 0.3218  \u001b[0m | \u001b[0m 1.085   \u001b[0m | \u001b[0m 6.268   \u001b[0m | \u001b[0m 1.347   \u001b[0m | \u001b[0m 0.8166  \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[665]\ttrain-auc:0.700933+0.00236211\ttest-auc:0.687762+0.00415006\n",
      "\n",
      "| \u001b[0m 2       \u001b[0m | \u001b[0m 0.6878  \u001b[0m | \u001b[0m 4.202   \u001b[0m | \u001b[0m 0.5478  \u001b[0m | \u001b[0m 5.619   \u001b[0m | \u001b[0m 6.005   \u001b[0m | \u001b[0m 9.193   \u001b[0m | \u001b[0m 0.6551  \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[250]\ttrain-auc:0.673725+0.00270497\ttest-auc:0.665721+0.00586694\n",
      "\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m 0.6657  \u001b[0m | \u001b[0m 3.535   \u001b[0m | \u001b[0m 0.7106  \u001b[0m | \u001b[0m 9.77    \u001b[0m | \u001b[0m 10.02   \u001b[0m | \u001b[0m 1.165   \u001b[0m | \u001b[0m 0.9316  \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[611]\ttrain-auc:0.78788+0.000904657\ttest-auc:0.714603+0.00585938\n",
      "\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m 0.7146  \u001b[0m | \u001b[0m 0.01395 \u001b[0m | \u001b[0m 0.4507  \u001b[0m | \u001b[0m 2.635   \u001b[0m | \u001b[0m 9.801   \u001b[0m | \u001b[0m 12.95   \u001b[0m | \u001b[0m 0.8445  \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[874]\ttrain-auc:0.753232+0.000981659\ttest-auc:0.712344+0.00517962\n",
      "\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m 0.7123  \u001b[0m | \u001b[0m 1.494   \u001b[0m | \u001b[0m 0.7328  \u001b[0m | \u001b[0m 2.164   \u001b[0m | \u001b[0m 5.159   \u001b[0m | \u001b[0m 13.84   \u001b[0m | \u001b[0m 0.8017  \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[128]\ttrain-auc:0.921693+0.00216906\ttest-auc:0.71045+0.00530195\n",
      "\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m 0.7105  \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 0.625   \u001b[0m | \u001b[0m 0.1498  \u001b[0m | \u001b[0m 8.261   \u001b[0m | \u001b[0m 16.15   \u001b[0m | \u001b[0m 0.9657  \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[55]\ttrain-auc:0.932968+0.00214413\ttest-auc:0.696534+0.00585321\n",
      "\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m 0.6965  \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 10.86   \u001b[0m | \u001b[0m 5.841   \u001b[0m | \u001b[0m 0.5     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[1368]\ttrain-auc:0.712389+0.00139844\ttest-auc:0.694447+0.0036985\n",
      "\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m 0.6944  \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 0.6235  \u001b[0m | \u001b[0m 6.272   \u001b[0m | \u001b[0m 8.528   \u001b[0m | \u001b[0m 17.51   \u001b[0m | \u001b[0m 0.5981  \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[294]\ttrain-auc:0.829903+0.00116729\ttest-auc:0.716384+0.00639328\n",
      "\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m 0.7164  \u001b[0m | \u001b[0m 6.141   \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[64]\ttrain-auc:0.969294+0.00142533\ttest-auc:0.703839+0.00434128\n",
      "\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m 0.7038  \u001b[0m | \u001b[0m 3.234   \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 14.74   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[340]\ttrain-auc:0.830957+0.00118719\ttest-auc:0.716299+0.00543992\n",
      "\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m 0.7163  \u001b[0m | \u001b[0m 7.435   \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 17.49   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[172]\ttrain-auc:0.8558+0.000651055\ttest-auc:0.70668+0.00746121\n",
      "\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m 0.7067  \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 9.866   \u001b[0m | \u001b[0m 20.0    \u001b[0m | \u001b[0m 0.5     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[332]\ttrain-auc:0.819404+0.0012599\ttest-auc:0.715153+0.00487531\n",
      "\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m 0.7152  \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 12.57   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[34]\ttrain-auc:0.670395+0.00284399\ttest-auc:0.66404+0.00422902\n",
      "\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m 0.664   \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 6.183   \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 19.81   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[120]\ttrain-auc:0.869175+0.00157052\ttest-auc:0.710859+0.00558416\n",
      "\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m 0.7109  \u001b[0m | \u001b[0m 5.189   \u001b[0m | \u001b[0m 0.5747  \u001b[0m | \u001b[0m 0.09182 \u001b[0m | \u001b[0m 8.654   \u001b[0m | \u001b[0m 12.95   \u001b[0m | \u001b[0m 0.6548  \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[110]\ttrain-auc:0.896027+0.00270849\ttest-auc:0.712411+0.0048575\n",
      "\n",
      "| \u001b[0m 16      \u001b[0m | \u001b[0m 0.7124  \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 11.25   \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[88]\ttrain-auc:0.847653+0.00030762\ttest-auc:0.702376+0.00679876\n",
      "\n",
      "| \u001b[0m 17      \u001b[0m | \u001b[0m 0.7024  \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 8.109   \u001b[0m | \u001b[0m 0.5     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[252]\ttrain-auc:0.794057+0.00058383\ttest-auc:0.709741+0.00576175\n",
      "\n",
      "| \u001b[0m 18      \u001b[0m | \u001b[0m 0.7097  \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 5.605   \u001b[0m | \u001b[0m 0.5     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[237]\ttrain-auc:0.838093+0.00195184\ttest-auc:0.715159+0.00684298\n",
      "\n",
      "| \u001b[0m 19      \u001b[0m | \u001b[0m 0.7152  \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 9.372   \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[689]\ttrain-auc:0.675009+0.00332771\ttest-auc:0.666605+0.00820713\n",
      "\n",
      "| \u001b[0m 20      \u001b[0m | \u001b[0m 0.6666  \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 13.8    \u001b[0m | \u001b[0m 0.5     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping. Best iteration:\n",
      "[156]\ttrain-auc:0.814832+0.0018914\ttest-auc:0.710452+0.00649244\n",
      "\n",
      "| \u001b[0m 21      \u001b[0m | \u001b[0m 0.7105  \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.5     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[51]\ttrain-auc:0.935216+0.00164439\ttest-auc:0.706881+0.00495284\n",
      "\n",
      "| \u001b[0m 22      \u001b[0m | \u001b[0m 0.7069  \u001b[0m | \u001b[0m 5.509   \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[313]\ttrain-auc:0.844097+0.00169582\ttest-auc:0.717143+0.00497509\n",
      "\n",
      "| \u001b[95m 23      \u001b[0m | \u001b[95m 0.7171  \u001b[0m | \u001b[95m 4.099   \u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 0.0     \u001b[0m | \u001b[95m 5.0     \u001b[0m | \u001b[95m 5.377   \u001b[0m | \u001b[95m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[121]\ttrain-auc:0.861537+0.000673411\ttest-auc:0.704227+0.00621222\n",
      "\n",
      "| \u001b[0m 24      \u001b[0m | \u001b[0m 0.7042  \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 20.0    \u001b[0m | \u001b[0m 0.5     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[90]\ttrain-auc:0.845593+0.00133256\ttest-auc:0.707456+0.00550936\n",
      "\n",
      "| \u001b[0m 25      \u001b[0m | \u001b[0m 0.7075  \u001b[0m | \u001b[0m 5.339   \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 8.853   \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 0.5     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[325]\ttrain-auc:0.851317+0.00159143\ttest-auc:0.716468+0.00503248\n",
      "\n",
      "| \u001b[0m 26      \u001b[0m | \u001b[0m 0.7165  \u001b[0m | \u001b[0m 2.632   \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 20.0    \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[50]\ttrain-auc:0.951022+0.00220192\ttest-auc:0.694021+0.0050604\n",
      "\n",
      "| \u001b[0m 27      \u001b[0m | \u001b[0m 0.694   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 20.0    \u001b[0m | \u001b[0m 0.5     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[48]\ttrain-auc:0.692049+0.00561654\ttest-auc:0.67706+0.00538116\n",
      "\n",
      "| \u001b[0m 28      \u001b[0m | \u001b[0m 0.6771  \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 3.623   \u001b[0m | \u001b[0m 15.0    \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[119]\ttrain-auc:0.879951+0.00179697\ttest-auc:0.7121+0.00592652\n",
      "\n",
      "| \u001b[0m 29      \u001b[0m | \u001b[0m 0.7121  \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 10.77   \u001b[0m | \u001b[0m 14.1    \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "Multiple eval metrics have been passed: 'test-auc' will be used for early stopping.\n",
      "\n",
      "Will train until test-auc hasn't improved in 50 rounds.\n",
      "Stopping. Best iteration:\n",
      "[79]\ttrain-auc:0.689448+0.0024672\ttest-auc:0.676428+0.00416538\n",
      "\n",
      "| \u001b[0m 30      \u001b[0m | \u001b[0m 0.6764  \u001b[0m | \u001b[0m 10.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 3.104   \u001b[0m | \u001b[0m 5.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 1.0     \u001b[0m |\n",
      "=================================================================================================\n"
     ]
    }
   ],
   "source": [
    "num_rounds = 3000\n",
    "random_state = 2021\n",
    "num_iter = 25\n",
    "init_points = 5\n",
    "params = {\n",
    "    'eta': 0.1,\n",
    "    'silent': 1,\n",
    "    'eval_metric': 'auc',\n",
    "    'verbose_eval': True,\n",
    "    'seed': random_state\n",
    "}\n",
    "\n",
    "xgbBO = BayesianOptimization(xgb_evaluate, {'min_child_weight': (1, 20),\n",
    "                                            'colsample_bytree': (0.1, 1),\n",
    "                                            'max_depth': (5, 15),\n",
    "                                            'subsample': (0.5, 1),\n",
    "                                            'gamma': (0, 10),\n",
    "                                            'alpha': (0, 10),\n",
    "                                            })\n",
    "\n",
    "xgbBO.maximize(init_points=init_points, n_iter=num_iter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
